MMS Ion Composition Study

Compare dispersion list

Purpose

In attampt to make sense of the automatically identified dispersion list and it's bias, we compare the results from the eye identified dispersions analysis results with the program - identified ones.

  • There are 47 MMS manual identification has match with the auto-identification (Not all of them are exact match)
  • Not all of them have a match. 7 of them do not have match.
  • 5 of them are too short.
  • Other 2:
    • 2017-04-06/19:08:00 (42 min) - gap in the dispersion
    • 2017-08-17/12:39:00 (56 min) - Too flat
In [6]:
images_array = ["idl_plots/dispersion_day/o_beam20170406_180000_to_20170407_000000_dispersion.png"
                ,"idl_plots/dispersion_day/o_beam20170817_120000_to_20170817_180000_dispersion.png"]
ipyplot.plot_images(images_array, img_width=350)

0

idl_plots/dispersion_day/o_beam20170406_180000_to_20170407_000000_dispersion.png

1

idl_plots/dispersion_day/o_beam20170817_120000_to_20170817_180000_dispersion.png

In [9]:
display_array = []
In [10]:
for i in toplot.index:
    display_array = []
    i_toplot = toplot.loc[i,:]
    match_date = lambda x: (i_toplot['date'][0:4]+i_toplot['date'][5:7]+i_toplot['date'][8:10]) in x
    match_list1 = list(map(match_date, dispersion_tplot_list))
    if match_list1.count(True) > 0:
        index1 = match_list1.index(True)
        i_tplot1 = dispersion_tplot_list[index1]
        display_array.append(i_tplot1)

    match_list2 = list(map(match_date, dispersion_fitting_plot_list))
    if match_list2.count(True) > 0:
        index2 = match_list2.index(True)
        i_tplot2 = dispersion_fitting_plot_list[index2]
        display_array.append(i_tplot2)

    ipyplot.plot_images(display_array, img_width=400)

# display(Image(url = display_array[0]))

0

idl_plots/dispersion_day/o_beam20160412_060000_to_20160412_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160412_074230_to_20160412_080730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160413_060000_to_20160413_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160413_064730_to_20160413_071230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160422_120000_to_20160422_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160422_143730_to_20160422_151230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160506_000000_to_20160506_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160506_051230_to_20160506_052730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160506_000000_to_20160506_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160506_051230_to_20160506_052730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160515_000000_to_20160515_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160515_043230_to_20160515_044730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160516_120000_to_20160516_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160516_143230_to_20160516_145230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160516_120000_to_20160516_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160516_143230_to_20160516_145230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160516_120000_to_20160516_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160516_143230_to_20160516_145230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160528_000000_to_20160528_060000_dispersion.png

0

idl_plots/dispersion_day/o_beam20160530_000000_to_20160530_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160530_061230_to_20160530_063730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160530_000000_to_20160530_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160530_061230_to_20160530_063730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160605_060000_to_20160605_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160605_084730_to_20160605_090230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160606_000000_to_20160606_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160606_014730_to_20160606_020230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160606_000000_to_20160606_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160606_014730_to_20160606_020230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160611_060000_to_20160611_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160611_120730_to_20160611_122230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160612_120000_to_20160612_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160612_144730_to_20160612_151730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160803_060000_to_20160803_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160803_095230_to_20160803_101730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160803_060000_to_20160803_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160803_095230_to_20160803_101730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160805_060000_to_20160805_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160805_091230_to_20160805_094230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160809_060000_to_20160809_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160809_094230_to_20160809_095730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160812_000000_to_20160812_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160812_055730_to_20160812_061230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160812_000000_to_20160812_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160812_055730_to_20160812_061230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160813_060000_to_20160813_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160813_082230_to_20160813_085230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160815_060000_to_20160815_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160815_073230_to_20160815_075730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160824_060000_to_20160824_120000_dispersion.png

1

idl_plots/dispersion/o_beam20160824_074230_to_20160824_081230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160901_000000_to_20160901_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160901_025230_to_20160901_030730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160903_120000_to_20160903_180000_dispersion.png

1

idl_plots/dispersion/o_beam20160903_170730_to_20160903_172730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160904_000000_to_20160904_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160904_062230_to_20160904_063730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160904_000000_to_20160904_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160904_062230_to_20160904_063730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160905_000000_to_20160905_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160905_020230_to_20160905_021730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160905_000000_to_20160905_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160905_020230_to_20160905_021730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20160908_000000_to_20160908_060000_dispersion.png

1

idl_plots/dispersion/o_beam20160908_043730_to_20160908_045230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20170802_060000_to_20170802_120000_dispersion.png

1

idl_plots/dispersion/o_beam20170802_092230_to_20170802_093730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20170824_120000_to_20170824_180000_dispersion.png

1

idl_plots/dispersion/o_beam20170824_151230_to_20170824_152730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20170830_060000_to_20170830_120000_dispersion.png

1

idl_plots/dispersion/o_beam20170830_080230_to_20170830_081730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20170902_000000_to_20170902_060000_dispersion.png

1

idl_plots/dispersion/o_beam20170902_033730_to_20170902_035230dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20170913_060000_to_20170913_120000_dispersion.png

1

idl_plots/dispersion/o_beam20170913_090730_to_20170913_092730dispersion_fitting.png

0

idl_plots/dispersion_day/o_beam20171011_000000_to_20171011_060000_dispersion.png

0

idl_plots/dispersion_day/o_beam20171025_060000_to_20171025_120000_dispersion.png

1

idl_plots/dispersion/o_beam20171025_081230_to_20171025_083730dispersion_fitting.png

In [11]:
i=0
In [12]:
fig = px.scatter(dispersion_list_m_merged, x="RE", y="estimated_distance_m", trendline="ols", color = "good_fit", size="Duration", hover_name="Event Time")
reference_line = go.Scatter(x=[0, 35], y=[0, 35], mode="lines", line=go.scatter.Line(color="gray"), showlegend=False)
fig.add_trace(reference_line)
fig.update_layout( title = 'Replicate manual identification',font=dict( family="Courier New, monospace",size=18    ), legend_x = 0, legend_y=1)
Discussion
  1. Many of the data are not included in the "good" dispersion list. We need to investigate each plots to find out the reason. The next step is to plot the start and end point in Manul list in the tplot
  2. The duration of the plots varies (8 min to 68 min). In contrast to auto-list (20min to 50min)
  3. It seems that there are two group of data.
In [13]:
fig = px.scatter(dispersion_list_m_merged.loc[index,], x="estimated_distance_m", y="estimated_distance"
                 , hover_name="Event Time" ,size="Duration", trendline="ols"
                 , hover_data = ["index", "BETA", "direction", "GSM_Z", "p_value", 'GSM_X', 'BX_GSM']
                , color = "good_fit"
                 )

reference_line = go.Scatter(x=[0, 35], y=[0, 35], mode="lines", line=go.scatter.Line(color="gray"), showlegend=False)

fig.add_trace(reference_line)

fig.update_layout(title = 'Comparison of estimated distance (auto vs manual)' ,
 font=dict(family="Courier New, monospace",size=18    ), legend_x = 1, legend_y=1)
Discussion

The auto process consistently makes bigger estimation of distance than manual process

In [14]:
fig = px.scatter(dispersion_list_m_merged.loc[index,], x="model_field_line_length_idl", y="estimated_distance_m"
                 , size="Duration"
                 , hover_name="Event Time"
                 , hover_data = ["index", "BETA", "direction", "GSM_Z", "p_value", 'GSM_X', 'BX_GSM']
                , trendline="ols"
                , color = "good_fit"
                 )

reference_line = go.Scatter(x=[0, 35], y=[0, 35], mode="lines", line=go.scatter.Line(color="gray"), showlegend=False)

fig.add_trace(reference_line)

fig.update_layout(title = 'Manual estiamted distance VS traced field line length ('+model+')' ,
 font=dict(family="Courier New, monospace",size=18    ), legend_x = 0, legend_y=1)

fig.show()
In [15]:
fig = px.scatter(dispersion_list_m_merged.loc[index,], x="model_field_line_length_idl", y="estimated_distance", size="dispersion_time", hover_name="TIME", hover_data = ["index", "BETA", "direction", "GSM_Z", "p_value", 'GSM_X', 'BX_GSM']
                , trendline="ols", color = "good_fit")
reference_line = go.Scatter(x=[0, 35], y=[0, 35], mode="lines", line=go.scatter.Line(color="gray"), showlegend=False)
fig.add_trace(reference_line)
fig.update_layout(title = 'Auto estiamted distance VS traced field line length ('+model+')', font=dict(family="Courier New, monospace",size=18), legend_x = 0, legend_y=1)
In [16]:
fig = px.scatter(dispersion_list_m_merged.loc[index,:], x="model_field_line_length_idl", y="estimated_distance_m"
                 , size="dispersion_time"
                 , hover_name="datetime_str"
                 , hover_data = ["index", "BETA", "direction", "GSM_Z", "p_value", 'GSM_X', 'BX_GSM','MLT_y']
                , trendline="ols"
                 , symbol = "direction_et"
          #       , facet_col="direction_et"
         #        , facet_row="region"
                , color = "region" #np.log10(dispersion_list_m_merged.loc[index,"mean_E"]), range_color=[3,5]
                 )

reference_line = go.Scatter(x=[0, 35], y=[0, 35], mode="lines", line=go.scatter.Line(color="gray"), showlegend=False)

fig.add_trace(reference_line)
#fig.add_trace(reference_line, row=1, col=1)
#fig.add_trace(reference_line, row=1, col=2)

fig.update_layout(#title = ' ' + ', goodness of fit > ' + str(goodness_of_fit_threshold),
 font=dict(
        family="Courier New, monospace",
        size=18    ), legend_x = 1, legend_y=1)

fig.update_layout(coloraxis_colorbar=dict(
    title="energy",
    tickvals=[1,2,3,4,5],
    ticktext=["10", "100", "1k","100k","100k"],
))